package com.frontalcode 
{
	import com.frontalcode.*;

    internal class Tokenizer extends OptionableObject
	{
		internal var debug : Boolean = false;
		
		internal var allTokens : Array;
		internal var allTokensHash : Object;
		internal var allSelfDelimitingTokensHash : Object;
		internal var selfDelimitingTokenCheck : String;
		
		private var tokenStream : Object;
		
		function Tokenizer ( tokens : Array, options : Object = null )
		{
			super ( options );
			
			setOptionsFromLoaderParameters ( );
			
			allTokens = tokens;
			
			// Sort the tokens by length so that when we match one we know we won't
			// have a better match later.
			//
			
			allTokens.sort ( 
				function ( a : Object, b : Object ) : int
				{ 
					if ( a.selfDelimiting && ! b.selfDelimiting ) return 1;
					if ( ! a.selfDelimiting && b.selfDelimiting ) return -1;
					
					if ( a.token.length < b.token.length ) return -1;
					if ( a.token.length > b.token.length ) return 1;
					
					return 0;
				}, Array.DESCENDING );
			
			// Create an index of the tokens.
			//
			
			allTokensHash = new Object ( );
			allSelfDelimitingTokensHash = new Object ( );
			selfDelimitingTokenCheck = "";
			
			for ( var i : int = 0; i < allTokens.length; i++ )
			{
				if ( allTokens [ i ].selfDelimiting )
				{
					allSelfDelimitingTokensHash [ allTokens [ i ].token ] = allTokens [ i ];
					
					if ( selfDelimitingTokenCheck.indexOf ( allTokens [ i ].token.charAt ( 0 ) ) < 0 )
					{
						selfDelimitingTokenCheck += allTokens [ i ].token.charAt ( 0 );
					}
				}
				
				allTokensHash [ allTokens [ i ].token ] = allTokens [ i ];
			}
		}
		
		/*
			Function: tokenizeCode
			
				Convert a code string into a stream of tokens.
				
			Returns:
			
				The tokenized code stream.
		*/
		internal function tokenizeCode ( code : String, debug : * ) : Object
		{
			throw new FrontalError ( "call of pure virtual function BASIK.Tokenizer.tokenizeCode" );
			
			return { };
		}
		
		/*
			Function: copyObject
			
				Return a copy of an Object. This is used for making copies of token
				definitions so we can store tokenizer state in them.
				
			Returns:
			
				The copied Object.
		*/
		internal function copyObject ( src : Object ) : Object
		{
			var result : Object;
			
			if ( src != null )
			{
				result = { };
				for ( var i : * in src ) result [ i ] = src [ i ];
			}
			
			return result;
		}
		
		/*
			The following functions add support for pretty debug traces and errors.
		*/
		
		internal function getCodeSnippet ( currentCode : String, currentStatementCodeIndex: Number, codeIndex : Number = NaN ) : String
		{
			var snippet : String = "";
			
			if ( isNaN ( codeIndex ) ) codeIndex = currentStatementCodeIndex;
			var scan : Number = codeIndex;
			var nextChar : String;
			
			if ( ! isNaN ( currentStatementCodeIndex ) )
			{
				scan = currentStatementCodeIndex;
			}
			else
			{
				while ( scan > 0 )
				{
					if ( ( nextChar = currentCode.charAt ( scan ) ) == "\r" || nextChar == "\n" || nextChar == ";" || nextChar == "}" ) break;
					scan--;
				}
			}
			
			snippet += currentCode.substring ( scan, codeIndex + 1 );
			
			scan = codeIndex + 1;
			while ( scan < currentCode.length - 1 )
			{
				if ( ( nextChar = currentCode.charAt ( scan ) ) == "\r" || nextChar == "\n" || nextChar == ";" || nextChar == "}" ) break;
				scan++;
				if ( ( nextChar = currentCode.charAt ( scan ) ) == ";" || nextChar == "}" ) break;
			}
			
			snippet += currentCode.substring ( scan, codeIndex + 1 );
			
			return snippet;
		}
	}
}

